// robotsCheck.js
const robotsParser = require('robots-parser');
const axios = require('axios');

async function isAllowed(url) {
  const base = new URL(url).origin;
  const robotsTxt = await axios.get(`${base}/robots.txt`);
  const robots = robotsParser(`${base}/robots.txt`, robotsTxt.data);
  return robots.isAllowed(url, '*');
}

module.exports = isAllowed;
